#!/usr/bin/env python
# -*- encoding=utf-8 -*-
'''
Created on 2018/7/21

@author: peulerosci
'''

import logging
import logging.config
import xml.etree.ElementTree as ET
from subprocess import *
import tempfile
import linecache
import os, re, sys, datetime
reload(sys)
sys.setdefaultencoding('utf8')

if not os.path.isdir("log"):
    os.mkdir("log")
logging.config.fileConfig("logger.conf")
logger = logging.getLogger("copy_jenkins")

BRANCH = os.getenv('GERRIT_BRANCH')
if BRANCH == 'next_net' or BRANCH == 'next_hack' or BRANCH == 'next_docker' or BRANCH == 'next_kernel' or BRANCH == 'next_openeuler':
    BRANCH = 'next'
REPO_DIR = '/usr1/jenkins/EulerOS_' + BRANCH
PROJECT = os.getenv('GERRIT_PROJECT')

error_msg = '''
Please check you changelog, must be:
* Week Mon Date Year Author<euleros@huawei.com> - 1:1.0-1.h1
- Type:security etc.
- ID:CVE-2007-3568
- SUG:restart etc.
- DESC:Describe the problem in detail

For detail: http://3ms.huawei.com/hi/group/2692269/thread_7684785.html?mapId=9465969&for_statistic_from=all_group_forum
'''

only_hw_release = []
has_hw_release = []

def shell_cmd(s_cmd_line, inmsg=None):
    p = Popen(s_cmd_line, stdin=PIPE, stdout=PIPE, stderr=PIPE, shell=True)
    if inmsg:
        p.stdin.write(inmsg)
    out, err = p.communicate()
    return p.returncode, out, err


def changelog_check(changelog):
    status = True
    # 检查是不是总共5行
    #line_num = changelog.split('\n')
    #if len(line_num) != 5:
    #    logger.error('Must be 5 lines in one changelog')
    #    status = False
    try:
        first_line = changelog.split('\n')[0]
        # 检查第一行是不是'* '开头
        if not re.match(r'^\* ', first_line):
            logger.error("First line must start with '* '")
            status = False

        # 检查第一行前4个字符是不是: 星期 月 日 年, 以及格式是否符合要求
        mydate = re.findall(r'^\* (\w+ \w+ \d+ \d+)', first_line)
        try:
            datetime.datetime.strptime(mydate[0], '%a %b %d %Y')
        except Exception, e:
            logger.error('%s : %s' % (Exception, e))
            logger.error('date must be: Week Mon Date Year')
            status = False

        # 检查第一行后面的字符 修改人姓名，邮箱，以及修改后的版本号 是否填写正确
        #print "xxxxxxxxxxxxxxxxxxxx" + str(re.match(r'.* \w+<.+@huawei.com> - (\w+(\.)*)*\w+-(\w+(\.)*)*\w+', first_line))
        if not re.match(r'.* \S+<.+@huawei.com> - (\w+(\.)*)*\w+-(\w+(\.)*)*\w+', first_line.replace(':','')):
            logger.error('Author&version must be: Author<xxx@huawei.com> - version-release')
            status = False

        second_line = changelog.split('\n')[1]
        # 检查第二行是不是 '- Type:' 开头的
        if not re.match(r'^- Type:\w+', second_line):
            logger.error("Second line must start with '- Type:', and there should be no space after :.")
            status = False

        # 检查更新类型是不是属于 'bugzillas,cves,enhancement,bugfix,security,severity,recommended,new-packages'
        types = 'bugzillas,cves,enhancement,bugfix,security,severity,recommended,new-packages'.split(',')
        type_value = second_line.replace(r'- Type:', '')
        type_value_list = type_value.split(';')
        for type_tmp in type_value_list:
            if type_tmp.strip() not in types:
                logger.error("The type must be one of these: {0}".format(types))
                status = False

        third_line = changelog.split('\n')[2]
        # 检查第三行是不是 '- ID:' 开头的
        if not re.match(r'^- ID:\w+', third_line):
            logger.error("Third line must start with '- ID:', and there should be no space after :. ; also, the value should not be black")
            status = False
        if type_value == 'cves':
            # 如果是cves类型的，检查ID的值是不是 CVE- 开头的
            id_value = third_line.replace(r'- ID:', '').split(',')
            for myid in id_value:
                if not re.match(r'^CVE-\w+', myid):
                    logger.error("For cves or security, the ID must start with 'CVE-'")
                    status = False

        fourth_line = changelog.split('\n')[3]
        # 检查第四行是不是 '- SUG:' 开头的
        if not re.match(r'^- SUG:\w+', fourth_line):
            logger.error("Third line must start with '- SUG:', and there should be no space after :. ; also, the value should not be black")
            status = False

        # 检查SUG是不是属于 'reboot,restart,NA'
        sugs = 'reboot,restart,NA'.split(',')
        sug_value = fourth_line.replace(r'- SUG:', '')
        if sug_value not in sugs:
            logger.error("type must be one of these: {0}".format(sugs))
            status = False

        fiveth_line = changelog.split('\n')[4]
        # 检查第五行是不是 '- DESC:' 开头的
        if not re.match(r'^- DESC:', fiveth_line):
            logger.error("Fiveth line must start with '- DESC:', and there should no space after :. ; also, the value should not be black")
            status = False
    except Exception, e:
        logger.error('%s : %s' % (Exception, e))
        status = False

    if not status:
        logger.error("{0}".format(error_msg))
        return False
    else:
        return True


# 有些spec写的不标准，缺少某些字段或多了某些不识别的字段会导致rpm -q --qf查询的时候失败，这个函数就是自动处理有问题的spec        
def deal_error_spec(spec, err):
    version_release = ''
    with open(spec, 'r') as my_f:
        tmpstr = my_f.read()
        a = tempfile.mktemp()
        with open(a, 'w') as b:
            b.write(tmpstr)
            b.flush()
    cmd = "rpm -q --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile"
    logger.info("Do cmd: {0}".format(cmd))
    while True:
        # 缺少Group字段的情况
        if re.match(r'^error: Group ', err):
            logger.warning('No Group in spec==================={0}'.format(err))
            with open(a, 'r') as my_f:
                tmpstr = my_f.read()
                tmpstr1 = re.sub(r'(%package|Name:)(.*)', r'\1\2\nGroup:xdf', tmpstr, re.M)
            os.remove(a)
            a = tempfile.mktemp()
            with open(a, 'w') as b:
                b.write(tmpstr1)
                b.flush()
            cmd = "rpm -q --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {0}".format(a)
            ret, out, err = shell_cmd(cmd)
            if ret == 0:
                os.remove(a)
                version_release = out.split('\n')[0]
                break
        
        # 存在有问题字段的情况
        elif re.match(r'.*error: line .*:', err, re.S):
            error_line_num = re.findall(r'error: line (.*?):.*', err)[0]
            logger.warning('spec error: ==================={0}\nline in spec: {1}'.format(err, error_line_num))
            error_line = linecache.getline(a, int(error_line_num))
            logger.warning('xxxxxxxxxxxxxxxxxxx: ==================={0}'.format(error_line))
            with open(a, 'r') as my_f:
                tmpstr = my_f.read()
                tmpstr1 = tmpstr.replace(error_line, '#'+error_line)
                #logger.warning('xxxxxxxxxxxxxxxxxxx: ==================={0}'.format(tmpstr1))
            os.remove(a)
            a = tempfile.mktemp()
            with open(a, 'w') as b:
                b.write(tmpstr1)
                b.flush()
            cmd = "rpm -q --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {0}".format(a)
            ret, out, err = shell_cmd(cmd)
            if ret == 0:
                os.remove(a)
                version_release = out.split('\n')[0]
                break
        # 存在有问题字段的情况
        elif re.match(r'.*error: .*\.spec:.*:', err, re.S):
            logger.warning('xxxxxxxxxxxxxxxxxxxxx============{0}'.format(err))
            error_line_num = re.findall(r'error: .*\.spec:(.*?):.*', err)[0]
            logger.warning('spec error: ==================={0}\nline in spec: {1}'.format(err, error_line_num))
            error_line = linecache.getline(a, int(error_line_num))
            logger.warning('xxxxxxxxxxxxxxxxxxx: ==================={0}'.format(error_line))
            with open(a, 'r') as my_f:
                tmpstr = my_f.read()
                tmpstr1 = tmpstr.replace(error_line, '#'+error_line)
                #logger.warning('xxxxxxxxxxxxxxxxxxx: ==================={0}'.format(tmpstr1))
            os.remove(a)
            a = tempfile.mktemp()
            with open(a, 'w') as b:
                b.write(tmpstr1)
                b.flush()
            cmd = "rpm -q --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {0}".format(a)
            ret, out, err = shell_cmd(cmd)
            if ret == 0:
                os.remove(a)
                version_release = out.split('\n')[0]
                break
        # 存在有问题字段的情况
        elif re.match(r'.*error: /tmp/.*:.*:', err, re.S):
            logger.warning('xxxxxxxxxxxxxxxxxxxxx============{0}'.format(err))
            error_line_num = re.findall(r'error: /tmp/.*:(.*?):.*', err)[0]
            logger.warning('spec error: ==================={0}\nline in spec: {1}'.format(err, error_line_num))
            error_line = linecache.getline(a, int(error_line_num))
            logger.warning('xxxxxxxxxxxxxxxxxxx: ==================={0}'.format(error_line))
            with open(a, 'r') as my_f:
                tmpstr = my_f.read()
                tmpstr1 = tmpstr.replace(error_line, '#'+error_line)
                #logger.warning('xxxxxxxxxxxxxxxxxxx: ==================={0}'.format(tmpstr1))
            os.remove(a)
            a = tempfile.mktemp()
            with open(a, 'w') as b:
                b.write(tmpstr1)
                b.flush()
            cmd = "rpm -q --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {0}".format(a)
            ret, out, err = shell_cmd(cmd)
            if ret == 0:
                os.remove(a)
                version_release = out.split('\n')[0]
                break
        # 存在不识别字段的情况
        elif re.match(r'^error:.*Unknown tag:', err):
            unknown_tag = re.findall(r'error:.*Unknown tag: (.*)', err)[0].split(' ')[0]
            logger.warning('unknown_tag==================={0}\ndetail error : {1}'.format(unknown_tag, err))
            with open(a, 'r') as my_f:
                tmpstr = my_f.read()
                tmpstr1 = re.sub(unknown_tag, '#'+unknown_tag, tmpstr)
            os.remove(a)
            a = tempfile.mktemp()
            with open(a, 'w') as b:
                b.write(tmpstr1)
                b.flush()
            cmd = "rpm -q --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {0}".format(a)
            ret, out, err = shell_cmd(cmd)
            if ret == 0:
                os.remove(a)
                version_release = out.split('\n')[0]
                break
        else:
            logger.error("run: %s : failed, the err is %s" % (cmd, err))
            out = 'read spec failed by: rpm -q --specfile'
            get_release_result = False
            sys.exit(1)
    return version_release


def version_transform(version):
    num = 0
    tmp1 = [0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0]
    tt = False
    first_num = True
    logger.info('version before transferm====={0}'.format(version))
    for i in version:
        if i == '.' or i == '-' or i == '_':
            num += 1
            tt = False
            continue
        if i.isalpha():
            if tt:
                num += 1
            if num != 0:
                tmp1[num] = i
            else:
                tmp1[0] = i
            num += 1
            tt = False
            continue
        tt = True
        if num == 0 and first_num:
            tmp1[num] = int(i)
            first_num = False
        else:
            tmp1[num] = tmp1[num]*10 + int(i)
    logger.info("version after transform=========={0}".format(tmp1))
    return tmp1

def check_spec_for_version(project_dir, spec_list, first_commitid):
    # 切换到对应的提交点
    cmd = 'cd {0};git checkout {1}'.format(project_dir, first_commitid)
    ret, out, err = shell_cmd(cmd)
    if ret != 0:
        logger.error("run: %s : failed,the out is %s,the err is %s" % (cmd, out, err))
        sys.exit(1)

    rpm_release = []
    for spec in spec_list:
        logger.info('spec is ====== {0}'.format(spec))
        # get release
        cmd = "rpm -q --define '_sourcedir {0}' --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {1}".format(
            project_dir,project_dir + '/' + spec)
        ret, out, err = shell_cmd(cmd)
        get_release_result = True
        if ret != 0:
            #tmp_version = deal_error_spec(project_dir + '/' + spec, err)
            #rpm_release.append(tmp_version)
            logger.error("run: %s : failed, the err is %s" % (cmd, err))
            get_release_result = False
            logger.info("For help:  http://3ms.huawei.com/hi/group/3078743/wiki_5601700.html ")
            sys.exit(1)
        else:
            rpm_release.append(out.split('\n')[0])

    # 切换到上一个提交点获取相关信息
    # 切换到对应的提交点
    cmd = 'cd {0};git checkout HEAD~1'.format(project_dir)
    ret, out, err = shell_cmd(cmd)
    if ret != 0:
        logger.error("run: %s : failed,the out is %s,the err is %s" % (cmd, out, err))
        sys.exit(1)
    ret1, out1, err1 = shell_cmd('cd {0};ls | egrep "*.spec"'.format(project_dir))
    if ret1 != 0:
        logger.info('This is the first commit of the module.')
        return True
    
    rpm_release_old = []
    for spec in spec_list:
        logger.info('spec is ====== {0}'.format(spec))
        # get release
        cmd = "rpm -q --define '_sourcedir {0}' --qf '%{{VERSION}}-%{{RELEASE}}\n' --specfile {1}".format(
            project_dir,project_dir + '/' + spec)
        ret, out, err = shell_cmd(cmd)
        get_release_result = True
        if ret != 0:
            tmp_version = deal_error_spec(project_dir + '/' + spec, err)
            rpm_release_old.append(tmp_version)
            #logger.error("run: %s : failed, the err is %s" % (cmd, err))
            #get_release_result = False
            #logger.info("For help:  http://3ms.huawei.com/hi/group/3078743/wiki_5601700.html ")
            #sys.exit(1)
        else:
            rpm_release_old.append(out.split('\n')[0])
    
    # 所有修改后的spec，version-release必须有升级，否则就失败
    version_change = []
    for i in range(0, len(rpm_release)):
        v = version_transform(rpm_release[i])
        v_old = version_transform(rpm_release_old[i])
        for j in range(0, len(v)):
            try:
                if v[j] > v_old[j]:
                    break
                elif v[j] == v_old[j]:
                    continue
                elif v[j] < v_old[j]:
                    logger.error("new version-release({0}) is small/equal than the last version-release({1})".format(rpm_release[i], rpm_release_old[i]))
                    version_change.append(False)
                    break
            except Exception, e:
                logger.error('%s : %s' % (Exception, e))
                logger.error("new version-release({0}) is small/equal than the last version-release({1})".format(rpm_release[i], rpm_release_old[i]))
                version_change.append(False)
                break
        if rpm_release[i] == rpm_release_old[i]:
            logger.error("new version-release({0}) is small/equal than the last version-release({1})".format(rpm_release[i], rpm_release_old[i]))
            version_change.append(False)
        else:
            version_change.append(True)
        logger.info("new: {0}  ###  old: {1}".format(rpm_release[i], rpm_release_old[i]))
        
        # 检查是不是所有的spec都只有hw-release的变化，如果是就需要检查patch的命名规范
        version_release = rpm_release[i].split('.h')[0]
        if len(rpm_release[i].split('.h')) == 1:
            only_hw_release.append(False)
            continue
        has_hw_release.append(True)
        hw_release = rpm_release[i].split('.h')[1]
        version_release_old = rpm_release_old[i].split('.h')[0]
        if len(rpm_release_old[i].split('.h')) == 1:
            hw_release_old = 0
        else:
            hw_release_old = rpm_release_old[i].split('.h')[1]
        if version_release != version_release_old or hw_release <= hw_release_old:
            only_hw_release.append(False)
            #logger.warning("xxxxxxxxxxxxxxxx==========={0}".format(only_hw_release))
    
    if False in version_change:
        return False
    else:
        return True

        
def check_spec(project_dir, spec_list):
    logger.info("start to check the spec")
    ret, out, err = shell_cmd('cd {0}; git rev-parse HEAD'.format(project_dir))
    if ret != 0:
        logger.error("run: %s : failed,the out is %s,the err is %s" % ('git', out, err))
        sys.exit(1)
    first_commitid = out
    
    # 检查spec的版本号变化
    if not check_spec_for_version(project_dir, spec_list, first_commitid):
        return False
    else:
        logger.info("The version-release for {0} is ok.".format(spec_list))
    
    # 检查spec的changelog变化，当spec中带huawei版本号的时候才检查这一项
    if True in has_hw_release:
        for spec in spec_list:
            logger.info('spec is ====== {0}'.format(spec))
            ret, out, err = shell_cmd('cd {0}; git checkout {1}'.format(project_dir, first_commitid))
            if ret != 0:
                logger.error("run: %s : failed,the out is %s,the err is %s" % ('git', out, err))
                sys.exit(1)
            with open(project_dir + '/' + spec) as myfile:
                new_spec = myfile.read()

            ret, out, err = shell_cmd('cd {0}; git checkout HEAD~1'.format(project_dir))
            if ret != 0:
                logger.error("run: %s : failed,the out is %s,the err is %s" % ('git', out, err))
                sys.exit(1)

            with open(project_dir + '/' + spec) as myfile:
                old_spec = myfile.read()

            new_changelog = re.findall('.*\n%changelog\s*\n(.+)', new_spec, re.S)
            if len(new_changelog) != 1:
                logger.error('There is no changelog in spec: {0}, num ==== {1}'.format(spec, len(new_changelog)))
                sys.exit(1)
            old_changelog = re.findall('.*\n%changelog\s*\n(.+)', old_spec, re.S)
            new_log = new_changelog[0].split('\n\n*')[0]
            #logger.info(str(new_log))
            logger.info('new changelog is ================= {0}'.format(new_log))
            if len(old_changelog) != 0 and new_log in old_changelog[0]:
                logger.error('You must add a new changelog when you modify the spec')
                sys.exit(1)

            if not changelog_check(new_log):
                return False
            else:
                logger.info("The changelog for {0} is ok.".format(spec))
    
    ret, out, err = shell_cmd('cd {0}; git checkout {1}'.format(project_dir, first_commitid))
    if ret != 0:
        logger.error("run: %s : failed,the out is %s,the err is %s" % ('git', out, err))
        sys.exit(1)
    
    return True
            
# hw release changed, so check the name of patch
def check_patch(patch_list, file_list, project_dir):
    logger.info("start to check the name of patch")
    if False not in only_hw_release:
        for patch in patch_list:
            patch = patch.split('/')[-1]
            if not re.match('(huawei|backport)-(bugfix|cve|CVE|feature|security|enhancement)-.*', patch):
                logger.error("The patch: {0} is unconfirmed to the standards.\n please refer to:  http://3ms.huawei.com/hi/group/3078743/wiki_5161285.html".format(patch))
                return False
            else:
                logger.info("The patch: {0} is ok".format(patch))
    else:
        logger.info('hw release has no changes, so skip check patch name')
    return True
    
def check_env_for_rpmmacros():
    unknown_tag_list = {'%rhel':7}
    for key, value in unknown_tag_list:
        cmd = 'if ! rpm --showrc | grep "{0}" &> /dev/null;then echo "{0} {1}" >> ~/.rpmmacros;fi'.format(key, value)
        ret, out, err = shell_cmd(cmd)
        if ret != 0:
            logger.error("run: %s : failed,the out is %s,the err is %s" % ('git', out, err))
            sys.exit(1)

def get_project_path():
    default_xml = REPO_DIR + '/.repo/manifest.xml'
    with open(default_xml, 'r') as myfile:
        mystr = myfile.read()
        tmppath = re.findall('<project path="(.+)"\s+name="{0}"'.format(PROJECT), mystr)
        if len(tmppath) == 1:
            mypath = tmppath[0]
            return mypath

        xml_list = re.findall(r'<include name="(.+)".*/>', mystr)
        mypath = ''
        for next_xml in xml_list:
            if next_xml == 'singleRAN.xml':
                continue
            with open(REPO_DIR + '/.repo/manifests/' + next_xml) as myxml:
                tmp_str = myxml.read()
                xml_list_sub = re.findall(r'<include name="(.+)".*/>', tmp_str)
                if len(xml_list_sub) == 0:
                    tmppath = re.findall('<project path="(.+)"\s+name="{0}"'.format(PROJECT), tmp_str)
                    if len(tmppath) == 1:
                        mypath = tmppath[0]
                        break
                else:
                    for tmp_xml in xml_list_sub:
                        with open(REPO_DIR + '/.repo/manifests/' + tmp_xml) as tmpxml:
                            sub_str = tmpxml.read()
                            tmppath = re.findall('<project path="(.+)"\s+name="{0}"'.format(PROJECT), sub_str)
                            if len(tmppath) == 1:
                                mypath = tmppath[0]
                                break
                if mypath != '':
                    break
        if mypath == '':
            logger.error('can not find this git project: {0}'.format(PROJECT))
            sys.exit(1)

        return mypath

if __name__ == "__main__":
    check_env_for_rpmmacros
    work_dir = os.getcwd()
    project_dir = REPO_DIR + '/' + get_project_path()    
    ret, out, err = shell_cmd('cd {0};git diff --name-status HEAD~1 HEAD~0'.format(project_dir))
    if ret != 0:
        logger.error("run: %s : failed,the out is %s,the err is %s" % ('git', out, err))
        sys.exit(1)
    spec_list = re.findall('M\s(.+\.spec)', out)
    spec_list_add = re.findall('A\s(.+\.spec)', out)
    patch_list = re.findall('A\s(.+\.patch)', out)
    file_list = re.findall('M\s(.+)', out)
    file_list.extend(re.findall('R\s(.+)', out))
    file_list.extend(re.findall('A\s(.+)', out))
    
    has_spec = True
    ret1, out1, err1 = shell_cmd("cd {0};find . -name  '*.spec' | egrep '*.spec'".format(project_dir))
    #ret1, out1, err1 = shell_cmd('cd {0};ls | egrep "*.spec"'.format(project_dir))
    logger.info('ret1====={0}\nout1======={1}\nerr1========{2}'.format(ret1, out1, err1))
    if ret1 != 0:
        logger.info('There is no spec in the git.')
        has_spec = False
    
    # 每次都修改代码都必须修改spec，升级版本号，否则就失败退出
    if has_spec == False:
        logger.error('This module has no spec, skip')
        sys.exit(0)
    
    # 每次都修改代码都必须修改spec，升级版本号，否则就失败退出
    if has_spec and len(spec_list) == 0 and len(spec_list_add) == 0:
        logger.error('You must modify spec when udpate')
        sys.exit(1)
        
    # 没有spec和patch变更的时候，直接退出检查。这个一般不会走到，在jenkins里面配置了只有patch和spec的变化才会运行该脚本
    if len(spec_list) == 0 and len(patch_list) == 0:
        logger.info('===============================No spec&patch changed====================================')
    # 如果有patch生成，但是没有修改spec，直接失败退出。这种情况必须有至少一个spec里面的version-release升级
    elif len(spec_list) == 0 and len(spec_list_add) == 0 and len(patch_list) > 0:
        logger.error('You must modify spec when add patch')
        sys.exit(1)
    else:
        logger.info('===============================spec check====================================')
        if not check_spec(project_dir, spec_list):
            sys.exit(1)
    sys.exit(0)
